import sys, gzip, json, math, nltk, MySQLdb, os, enchant
from collections import Counter, defaultdict
#from MySQLdb import cursors

def calculate_TFIDF(TFs,IDFs):
	RES = Counter()
	for word in TFs:
		RES[word] = TFs[word] * IDFs[word]
	return RES

def tfidf_vector(valid,tf,idf):
	res = []
	for word in valid:
		res.append(tf[word] * idf[word])
	return res

if __name__ == "__main__":
	#conn = MySQLdb.connect(host="mysql.stalidis.com", user="ict4growth", passwd="stanford", db="ict4growth")
	#writer = conn.cursor()
	outfile = gzip.open("IDFmatrix.json.gz",'r')
	IDF = Counter(json.loads(outfile.readline()))
	outfile.close()
	VALID = sorted([w for w in IDF if all([p.isalpha() for p in w.split("-")])])
	#existing = os.listdir("TFMATRIX")
	existing = [f for f in os.listdir("TFMATRIX") if not f in os.listdir("VECTORS")]
	iterator = 0
	for TFfile in existing:
		iterator += 1
		tffile = gzip.open("TFMATRIX/"+TFfile,'r')
		TF = Counter(json.loads(tffile.readline()))
		tffile.close()
		tfidfs = gzip.open("VECTORS/"+TFfile,'w')
		tfidfs.write(json.dumps(tfidf_vector(VALID,TF,IDF))+"\n")
		tfidfs.close()
		#writer.execute("""INSERT INTO Books VALUES (%s,%s,%s)""",(TFfile.split(".")[0],json.dumps(TF),json.dumps(tfidf_vector(VALID,TF,IDF))))
		print "Calculated ", iterator, " documents"
	#conn.commit()
	#writer.close()
	#conn.close()
	sys.exit()


